package tw.com.hitevision.whiteboard.android.Crawler;

import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.media.MediaScannerConnection;
import android.net.Uri;
import android.os.Environment;
import android.util.Log;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.StringEscapeUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.*;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import androidx.annotation.NonNull;
import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import tw.com.hitevision.whiteboard.android.Props;

public class BrowserCrawler {
    private final static String TAG = BrowserCrawler.class.getSimpleName();
    public final static String DISCONNECTION_ERRMSG = "Can't connect to the network.";
    public final static String TIMEOUT_ERRMSG = "Connection timeout";
    private final static String OKHTTP_CANCEL_TAG = "stop";
    private int TIMEOUT = 5000; // milliseconds
    private int TIMEOUT_UNIT = 5000; // milliseconds
    private int MAX_TIMEOUT = 10000; // milliseconds
    private static final int BING_RESULTS_MAX_SIZE = 100;
    private static final int CHROME_RESULTS_MAX_SIZE = 100;
//    private static final String CHROME_AGENT = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36";
    private static final String CHROME_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36";
    //    private static final String BING_AGENT = "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko";
    private static final String BING_AGENT = "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)";
    //    private static final String BAIDU_AGENT = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
    private static final String BAIDU_AGENT = "Baiduspider-image+(+http://www.baidu.com/search/spider.htm)";
    private static final String HEIGHT_START = "&exph=";
    private static final String WIDTH_START = "&expw=";
    private static final String SEARCH_START = "&q=";


    private HashMap<Character, Character> charMap;
    private List<String> thumbnailsUrl;
    private List<String> sourceUrl;
    private List<float[]> imageSize;
    private boolean looped;
    private String errorMsg;
    private HttpURLConnection httpURLConnection;
    private InputStream httpInputStream;
    private OkHttpClient okHttpClient;

    private Context context;

    public BrowserCrawler(Context context) {
        this();
        this.context = context;
    }

    public BrowserCrawler() {
        sourceUrl = new ArrayList<>();
        thumbnailsUrl = new ArrayList<>();
        imageSize = new ArrayList<>();
        looped = false;
        okHttpClient = new OkHttpClient();
        okHttpClient.newBuilder()
                .connectTimeout(TIMEOUT, TimeUnit.MILLISECONDS)
                .readTimeout(TIMEOUT, TimeUnit.MILLISECONDS)
                .writeTimeout(TIMEOUT, TimeUnit.MILLISECONDS)
                .build();


        if (Props.getImageSearchBrowser() == Props.Browser.BAIDU) {
            buildBaiduTable();
        }
    }

    private void buildBaiduTable() {
        charMap = new HashMap<Character, Character>();
        charMap.put('w', 'a');
        charMap.put('k', 'b');
        charMap.put('v', 'c');
        charMap.put('1', 'd');
        charMap.put('j', 'e');
        charMap.put('u', 'f');
        charMap.put('2', 'g');
        charMap.put('i', 'h');
        charMap.put('t', 'i');
        charMap.put('3', 'j');
        charMap.put('h', 'k');
        charMap.put('s', 'l');
        charMap.put('4', 'm');
        charMap.put('g', 'n');
        charMap.put('5', 'o');
        charMap.put('r', 'p');
        charMap.put('q', 'q');
        charMap.put('6', 'r');
        charMap.put('f', 's');
        charMap.put('p', 't');
        charMap.put('7', 'u');
        charMap.put('e', 'v');
        charMap.put('o', 'w');
        charMap.put('8', '1');
        charMap.put('d', '2');
        charMap.put('n', '3');
        charMap.put('9', '4');
        charMap.put('c', '5');
        charMap.put('m', '6');
        charMap.put('0', '7');
        charMap.put('b', '8');
        charMap.put('l', '9');
        charMap.put('a', '0');
    }

    public List<String> getSourceUrl() {
        return sourceUrl;
    }

    public List<String> getThumbnailsUrl() {
        return thumbnailsUrl;
    }

    public List<float[]> getImageSize() {
        return imageSize;
    }

    public void searchSrcUrls(String searchTerm) throws Exception {
        if (Locale.getDefault().toString().equals("zh_CN")) {
            if (charMap == null) {
                buildBaiduTable();
            }
            searchSrcUrlsByBaidu(searchTerm);
            return;
        }

        if (Props.getImageSearchBrowser() == Props.Browser.CHROME) {
//            searchSrcUrlsByChrome(searchTerm);
            searchSrcUrlsByChromeAfter2020(searchTerm);
        } else if (Props.getImageSearchBrowser() == Props.Browser.BING) {
            searchSrcUrlsByBing(searchTerm);
        } else if (Props.getImageSearchBrowser() == Props.Browser.BAIDU) {
            searchSrcUrlsByBaidu(searchTerm);
        } else {
            searchSrcUrlsByChrome(searchTerm);
        }
    }

    private void searchSrcUrlsByBing(String searchTerm) throws Exception {
        // can grab at most 150 results
        String userAgent = BING_AGENT;
        String query = "https://www.bing.com/images/async?async=content&q=" + searchTerm + "&first=0&count=" + BING_RESULTS_MAX_SIZE + "&SafeSearch=off&adlt=off";
//        String query = "https://www.bing.com/images/search?q=" + searchTerm + "&view=detail&first=0&count=100";

        int max;
        int count;
        int lengthHeightStart = HEIGHT_START.length();
        int lengthWidthStart = WIDTH_START.length();

        if (sourceUrl != null && sourceUrl.size() > 0) {
            sourceUrl.clear();
        }

        if (thumbnailsUrl != null && thumbnailsUrl.size() > 0) {
            thumbnailsUrl.clear();
        }

        if (imageSize != null && imageSize.size() > 0) {
            imageSize.clear();
        }

        errorMsg = "";

        try {
//            Document doc = Jsoup.connect(query).timeout(TIMEOUT).userAgent(userAgent).referrer("https://www.bing.com/").get();
            String body = getHtmlViaOkHttp(query, userAgent);
            Document doc = Jsoup.parse(body);

            // debug
            saveHtmlToSdCard(doc);

            Elements elements = doc.select("div.imgpt a");

            count = 0;
            max = Math.min(100, BING_RESULTS_MAX_SIZE);
            JSONObject jsonObject;
            for (Element element : elements) {
                if (element.childNodeSize() > 0 && count < max) {
                    String href = element.attr("href");
                    int hStart = href.indexOf(HEIGHT_START);
                    int wStart = href.indexOf(WIDTH_START);
                    int sStart = href.indexOf(SEARCH_START);
                    String h = href.substring(hStart + lengthHeightStart, wStart);
                    String w = href.substring(wStart + lengthWidthStart, sStart);

                    int width = Integer.valueOf(w);
                    int height = Integer.valueOf(h);

                    String m = element.attr("m");
                    jsonObject = new JSONObject(m);
                    String t = (String) jsonObject.get("turl");
                    String s = (String) jsonObject.get("murl");

                    thumbnailsUrl.add(t);
                    sourceUrl.add(s);
                    imageSize.add(new float[] {width, height});
                    count++;
                }
            }

            looped = false;

            if (count == 0) {
                errorMsg = "No result";
            }

        } catch (SocketTimeoutException e) {
            e.printStackTrace();
            log("SocketTimeoutException Error connection in chrome crawler [" + query + "] : " + e);
            errorMsg = e.toString();
        } catch (IOException e) {
            e.printStackTrace();
            log("IOException Error connection in chrome crawler [" + query + "] : " + e);
            errorMsg = e.toString();
        }
    }

    private void searchSrcUrlsByChrome(String searchTerm) throws Exception {
        // can only grab first 100 results
        String userAgent = CHROME_AGENT;
        String query = "http://www.google.com/search?tbm=isch&q=" + searchTerm;

        boolean success = false;
        int max;
        int count;

        if (sourceUrl != null && sourceUrl.size() > 0) {
            sourceUrl.clear();
        }

        if (thumbnailsUrl != null && thumbnailsUrl.size() > 0) {
            thumbnailsUrl.clear();
        }

        if (imageSize != null && imageSize.size() > 0) {
            imageSize.clear();
        }

        errorMsg = "";

        try {
//            Document doc = Jsoup.connect(query).timeout(TIMEOUT).userAgent(userAgent).referrer("https://www.google.com/").get();

//            httpInputStream = getInputStreamViaHttpURLConnection(query);
//            Document doc = Jsoup.parse(httpInputStream, null, query);
            String body = getHtmlViaOkHttp(query, userAgent);
            Document doc = Jsoup.parse(body);

            // debug
//            if (isExternalStorageWritable()) {
//                // writable
//                externalPublicCreateFolder("test.html", body);
//            } else if (isExternalStorageReadable()) {
//                log("Permission readable only!!");
//            } else {
//                log("U shall not pass.");
//            }

            Elements elements = doc.select("div.rg_meta");

            count = 0;
            max = Math.min(100, CHROME_RESULTS_MAX_SIZE);
            JSONObject jsonObject;
            for (Element element : elements) {
                if (element.childNodeSize() > 0 && count < max) {
                    jsonObject = new JSONObject(element.childNode(0).toString());
                    int width = (int) jsonObject.get("ow");
                    int height = (int) jsonObject.get("oh");
                    String t = (String) jsonObject.get("tu");
                    String s = (String) jsonObject.get("ou");

                    thumbnailsUrl.add(t);
                    sourceUrl.add(s);
                    imageSize.add(new float[] {width, height});

                    count++;
                }
            }

            looped = false;
            success = true;

            if (count == 0) {
                errorMsg = "No result";
            }
//            abortHttpRequest();

        } catch (SocketTimeoutException e) {
            e.printStackTrace();
            log("SocketTimeoutException Error connection in chrome crawler [" + query + "] : " + e);
            errorMsg = translateErrMsg(e.toString());
        } catch (IOException e) {
            e.printStackTrace();
            log("IOException Error connection in chrome crawler [" + query + "] : " + e);
            errorMsg = translateErrMsg(e.toString());
        } finally {
//            if (!success) {
//                if (TIMEOUT<MAX_TIMEOUT) {
//                    TIMEOUT+=TIMEOUT_UNIT;
//                }
//                if (looped) {
//                    looped = false;
//                    return;
//                }
//                looped = true;
//                searchSrcUrls(searchTerm);
//            }
        }
    }

    private void searchSrcUrlsByChromeAfter2020(String searchTerm) {
        // can only grab first 100 results
        String query = "http://www.google.com/search?tbm=isch&q=" + searchTerm;

        int count = 0;

        if (sourceUrl != null && sourceUrl.size() > 0) {
            sourceUrl.clear();
        }

        if (thumbnailsUrl != null && thumbnailsUrl.size() > 0) {
            thumbnailsUrl.clear();
        }

        if (imageSize != null && imageSize.size() > 0) {
            imageSize.clear();
        }

        errorMsg = "";

        try {
            String body = getHtmlViaOkHttp(query, CHROME_AGENT);

//            // debug
//            if (isExternalStorageWritable()) {
//                // writable
//                externalPublicCreateFolder("test.html", body);
//            } else if (isExternalStorageReadable()) {
//                log("Permission readable only!!");
//            } else {
//                log("U shall not pass.");
//            }

            String temp0 = body;    //body.substring(body.indexOf("b-GRID_STATE0"));

            int maxSearchCount = 250;

            int fromIndex = 0;

            for (int i = 0;i < maxSearchCount;i++) {
                int tb0 = temp0.indexOf("http", fromIndex);
                fromIndex = tb0 + 1;
                int tb1 = temp0.indexOf('"', fromIndex);
                fromIndex = tb1 + 1;

                int sc0 = temp0.indexOf("http", fromIndex);
                fromIndex = sc0 + 1;
                int sc1 = temp0.indexOf('"', fromIndex);
                fromIndex = sc1 + 1;
                int sc2 = temp0.indexOf(',', fromIndex);
                fromIndex = sc2 + 1;
                int sc3 = temp0.indexOf(',', fromIndex);
                fromIndex = sc3 + 1;
                int sc4 = temp0.indexOf(']', fromIndex);
                fromIndex = sc4 + 1;

                fromIndex = temp0.indexOf("\"2008\"", fromIndex);

                if (fromIndex == -1) {
                    break;
                }

                try {
                    String tbn = StringEscapeUtils.unescapeJava(temp0.substring(tb0, tb1));
                    String src = temp0.substring(sc0, sc1);
                    int srcSize = src.length();

                    if (src.substring(srcSize - 3, srcSize).equalsIgnoreCase("gif")) {
                        src = tbn;
                    }

                    float height = Integer.parseInt(temp0.substring(sc2 + 1, sc3).trim());
                    float width = Integer.parseInt(temp0.substring(sc3 + 1, sc4).trim());

                    thumbnailsUrl.add(tbn);
                    sourceUrl.add(src);
                    imageSize.add(new float[] {width, height});

                    temp0 = temp0.substring(fromIndex);
                    fromIndex = 0;
                    count++;

                } catch (NumberFormatException e) {
                    e.printStackTrace();
                }

            }

            looped = false;

            if (count == 0) {
                errorMsg = "No result";
            }
//            abortHttpRequest();

        } catch (SocketTimeoutException e) {
            e.printStackTrace();
            log("SocketTimeoutException Error connection in chrome crawler [" + query + "] : " + e);
            errorMsg = translateErrMsg(e.toString());
        } catch (IOException e) {
            e.printStackTrace();
            log("IOException Error connection in chrome crawler [" + query + "] : " + e);
            errorMsg = translateErrMsg(e.toString());
        }
    }

    private void searchSrcUrlsByBaidu(String searchTerm) throws Exception {
        String userAgent = CHROME_AGENT;
//        String userAgent = BAIDU_AGENT;

        if (sourceUrl != null && sourceUrl.size() > 0) {
            sourceUrl.clear();
        }

        if (thumbnailsUrl != null && thumbnailsUrl.size() > 0) {
            thumbnailsUrl.clear();
        }

        if (imageSize != null && imageSize.size() > 0) {
            imageSize.clear();
        }
        baiduSearchPage(searchTerm, userAgent, 0);
        baiduSearchPage(searchTerm, userAgent, 1);

    }

    private void baiduSearchPage(String searchTerm, String userAgent, int pageIndex) throws Exception {
        String query = "http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&fp=result&queryWord=" +
                searchTerm +
                "&cl=2&lm=-1&ie=utf-8&oe=utf-8&st=-1&ic=0&word=" +
                searchTerm +
                "&face=0&istype=2nc=1&pn=" +
                pageIndex * 60 +
                "&rn=60";// number of item per page, 60 is the maximum number

        int count;

        errorMsg = "";

        try {
            String body = getHtmlViaOkHttp(query, userAgent);
            JSONObject reader = new JSONObject(body);
            JSONArray objects = reader.getJSONArray("data");
            objects.remove(objects.length() - 1);// remove last one, last one is an empty json

            count = 0;

            for (int i = 0; i < objects.length(); i++) {
                Object object = objects.get(i);
                JSONObject jsonObject = (JSONObject) object;
                int isGif = jsonObject.getInt("is_gif");
                if (isGif > 0) {
                    continue;
                }
                int width = jsonObject.getInt("width");
                int height = jsonObject.getInt("height");
                String objURL = jsonObject.getString("objURL");
                String thumbURL = jsonObject.getString("thumbURL");
                if (thumbURL != null) {
                    int id = thumbURL.indexOf("/u=");
                    thumbURL = "https://ss0.bdstatic.com/70cFvHSh_Q1YnxGkpoWK1HF6hhy/it" + thumbURL.subSequence(id, thumbURL.length());
                }
                thumbnailsUrl.add(thumbURL);
                sourceUrl.add(baiduUrlTranslator(objURL));
                imageSize.add(new float[] {width, height});
                count++;
            }


            looped = false;

            if (count == 0) {
                errorMsg = "No result";
            }

        } catch (SocketTimeoutException e) {
            e.printStackTrace();
            log("SocketTimeoutException Error connection in chrome crawler [" + query + "] : " + e);
            errorMsg = translateErrMsg(e.toString());
        } catch (IOException e) {
            e.printStackTrace();
            log("IOException Error connection in chrome crawler [" + query + "] : " + e);
            errorMsg = translateErrMsg(e.toString());
        }
    }

    private String baiduUrlTranslator(@NonNull String input) {
        String output = input;

        output = output.replace("_z2C$q", ":");
        output = output.replace("_z&e3B", ".");
        output = output.replace("AzdH3F", "/");
        output = output.replace("_z2C$q", ":");

        StringBuilder builder = new StringBuilder(output);

        for (int i = 0; i < builder.length(); i++) {
            Character c = charMap.get(builder.charAt(i));
            if (c != null) {
                builder.setCharAt(i, c);
            }
        }

        return builder.toString();
    }

    private String translateErrMsg(String origin) {
        if (origin != null && !origin.isEmpty()) {
            if (origin.contains("SocketTimeoutException")) {
                return TIMEOUT_ERRMSG;
            } else if (origin.contains("Read timed out")) {
                return TIMEOUT_ERRMSG;
            } else if (origin.contains("Unable to resolve host")) {
                return DISCONNECTION_ERRMSG;
            } else {
                log("Unexpected error msg : " + origin);
            }
        }
//        return origin;
        return null;
    }

    private InputStream getInputStreamViaHttpURLConnection(String query) throws IOException {
        URL url = new URL(query);
        log("HttpURLConnection : httpURLConnection = (HttpURLConnection) url.openConnection();");
        httpURLConnection = (HttpURLConnection) url.openConnection();
        log("HttpURLConnection : httpURLConnection.setConnectTimeout(TIMEOUT);");
        httpURLConnection.setConnectTimeout(TIMEOUT);
        httpURLConnection.setReadTimeout(TIMEOUT);
        httpURLConnection.setDoInput(true);
        httpURLConnection.connect();
        log("HttpURLConnection : InputStream in = httpURLConnection.getInputStream();");
        InputStream in = httpURLConnection.getInputStream();
        log("HttpURLConnection : finished");
        return in;
    }

    //    private ResponseBody getHtmlViaOkHttp(String url) throws IOException {
    private String getHtmlViaOkHttp(String url, String userAgent) throws IOException {
        Request request = new Request.Builder().tag(OKHTTP_CANCEL_TAG)
                .removeHeader("User-Agent").addHeader("User-Agent", userAgent)
                .addHeader("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2")
                .addHeader("Connection", "keep-alive")
                .addHeader("Upgrade-Insecure-Requests", "1")
                .url(url)
                .build();

        Response response = okHttpClient.newCall(request).execute();
        return response.body().string();
    }

    public void abortHttpRequest() {
        try {
            if (httpURLConnection != null && httpInputStream != null) {
                httpInputStream.close();
                httpURLConnection.disconnect();
            }
        } catch (IOException e) {
            e.printStackTrace();
            errorMsg = translateErrMsg(e.toString());
        }
    }

    public void abortOkHttpRequest() {
        for (Call call : okHttpClient.dispatcher().queuedCalls()) {
            if (call.request().tag().equals(OKHTTP_CANCEL_TAG)) {
                call.cancel();
            }
        }
        for (Call call : okHttpClient.dispatcher().runningCalls()) {
            if (call.request().tag().equals(OKHTTP_CANCEL_TAG)) {
                call.cancel();
            }
        }
    }

    public Bitmap getBitmapFromURL(String src) {
        try {
            URL url = new URL(src);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.setConnectTimeout(TIMEOUT);
            connection.setReadTimeout(TIMEOUT);
            connection.setDoInput(true);
            connection.connect();
            InputStream input = connection.getInputStream();
            return BitmapFactory.decodeStream(input);
        } catch (IOException e) {
            e.printStackTrace();
            log("IOException : " + e);
            return null;
        }

    }

    public boolean isExternalStorageWritable() {
        String state = Environment.getExternalStorageState();
        if (Environment.MEDIA_MOUNTED.equals(state)) {
            return true;
        }
        return false;
    }

    public boolean isExternalStorageReadable() {
        String state = Environment.getExternalStorageState();
        if (Environment.MEDIA_MOUNTED.equals(state) ||
                Environment.MEDIA_MOUNTED_READ_ONLY.equals(state)) {
            return true;
        }
        return false;
    }

    private void externalPublicCreateFolder(String fileName, String data) {
        String root = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES).toString();
        File myDir = new File(root + "/test_files/");
        if (!myDir.exists()) {
            myDir.mkdirs();
        }
        File file = new File(myDir, "20200210" + fileName);
        if (file.exists()) {
            file.delete();
        }

        try {
            FileOutputStream out = new FileOutputStream(file);
            out.write(data.getBytes());
            out.flush();
            out.close();
        } catch (Exception e) {
            e.printStackTrace();
            log("Error : " + e.getMessage());
        }
        // Tell the media scanner about the new file so that it is
        // immediately available to the user.
        MediaScannerConnection.scanFile(context, new String[] {file.toString()}, null,
                new MediaScannerConnection.OnScanCompletedListener() {
                    public void onScanCompleted(String path, Uri uri) {
                        Log.i("ExternalStorage", "Scanned " + path + ":");
                        Log.i("ExternalStorage", "-> uri=" + uri);
                    }
                });

    }

    private void saveHtmlToSdCard(Document document) {
        saveHtmlToSdCard(document.toString());
    }

    private void saveHtmlToSdCard(String doc) {
        if (isExternalStorageWritable()) {
            // writable
            externalPublicCreateFolder("test.html", doc);
        } else if (isExternalStorageReadable()) {
            log("Permission readable only!!");
        } else {
            log("U shall not pass.");
        }
    }

    public String getErrorMsg() {
        return errorMsg;
    }

    public void setErrorMsg(String s) {
        errorMsg = s;
    }

    private void log(String str) {
        Log.d(TAG, str);
    }
}